"""
Convert MoE models back to dense models.
"""
import json
import re
import shutil
from pathlib import Path

import torch
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME

# from transformers.models.llama import LlamaForCausalLM

# from smoe.models.llama_moe import LlamaMoEForCausalLM


def get_layer_nums(keys):
    layers = []
    for key in keys:
        s = re.search(r"model\.layers\.(\d+)\.", key)
        if s:
            layers.append(int(s.group(1)))
    layers.sort()
    return layers


def get_num_experts(keys):
    num_experts = 0
    for key in keys:
        s = re.search(
            r"model\.layers\.\d+\.mlp\.calculator\.experts\.weight_up\.(\d+)", key
        )
        if s:
            num_experts = max(num_experts, int(s.group(1)))
    num_experts += 1
    return num_experts


def main():
    dense_model_dir = "/mnt/petrelfs/share_data/quxiaoye/models/llama2_7B/"
    moe_model_dir = "/mnt/petrelfs/zhutong/smoe/outputs/random_split_scale4_112gpus_11900steps/checkpoint-11900/"
    out_dense_model_dir = "/mnt/petrelfs/zhutong/smoe/outputs/random_split_scale4_112gpus_11900steps_dense"
    filenames = ["pytorch_model-00001-of-00002.bin", "pytorch_model-00002-of-00002.bin"]

    out_dir = Path(out_dense_model_dir)
    out_dir.mkdir(exist_ok=True, parents=True)
    total_size = 0
    weight_map = {}
    for name in filenames:
        dense = {}
        moe = torch.load(Path(moe_model_dir) / name, map_location="cpu")
        for key in ["model.embed_tokens.weight", "model.norm.weight", "lm_head.weight"]:
            if key in moe:
                dense[key] = moe[key]
        layers = get_layer_nums(moe.keys())
        num_experts = get_num_experts(moe.keys())
        for layer_idx in layers:
            for key in ["q_proj", "k_proj", "v_proj", "o_proj"]:
                dense[f"model.layers.{layer_idx}.self_attn.{key}.weight"] = moe[
                    f"model.layers.{layer_idx}.self_attn.{key}.weight"
                ]
            dense[f"model.layers.{layer_idx}.self_attn.rotary_emb.inv_freq"] = moe[
                f"model.layers.{layer_idx}.self_attn.rotary_emb.inv_freq"
            ]
            dense[f"model.layers.{layer_idx}.input_layernorm.weight"] = moe[
                f"model.layers.{layer_idx}.input_layernorm.weight"
            ]
            dense[f"model.layers.{layer_idx}.post_attention_layernorm.weight"] = moe[
                f"model.layers.{layer_idx}.post_attention_layernorm.weight"
            ]

            up_proj = []
            for expert_idx in range(num_experts):
                param = moe[
                    f"model.layers.{layer_idx}.mlp.calculator.experts.weight_up.{expert_idx}"
                ]
                up_proj.append(param)
            up_proj_cat = torch.cat(up_proj, dim=0)
            dense[f"model.layers.{layer_idx}.mlp.up_proj.weight"] = up_proj_cat

            gate_proj = []
            for expert_idx in range(num_experts):
                param = moe[
                    f"model.layers.{layer_idx}.mlp.calculator.experts.weight_gate.{expert_idx}"
                ]
                gate_proj.append(param)
            gate_proj_cat = torch.cat(gate_proj, dim=0)
            dense[f"model.layers.{layer_idx}.mlp.gate_proj.weight"] = gate_proj_cat

            down_proj = []
            for expert_idx in range(num_experts):
                param = moe[
                    f"model.layers.{layer_idx}.mlp.calculator.experts.weight_down.{expert_idx}"
                ]
                down_proj.append(param)
            down_proj_cat = torch.cat(down_proj, dim=1)
            dense[f"model.layers.{layer_idx}.mlp.down_proj.weight"] = down_proj_cat

        torch.save(dense, out_dir / name)

        for p_key, param in dense.items():
            total_size += param.numel() * dtype_byte_size(param.dtype)
            weight_map[p_key] = name

    index_map = {"metadata": {"total_size": total_size}, "weight_map": weight_map}
    with out_dir.joinpath(WEIGHTS_INDEX_NAME).open("w", encoding="utf8") as fout:
        json.dump(index_map, fout, indent=2, ensure_ascii=False)

    # copy files from original dense model to target model
    filenames = [
        "config.json",
        "generation_config.json",
        "special_tokens_map.json",
        "tokenizer_config.json",
        "tokenizer.model",
    ]
    for filename in filenames:
        shutil.copy(Path(dense_model_dir) / filename, out_dir / filename)


if __name__ == "__main__":
    main()
    # x = torch.load("outputs/random_split_scale4_112gpus_11900steps_dense/pytorch_model-00001-of-00002.bin")
    # for name, param in x.items():
    #     print(name, param.shape)


"""
dense

model.embed_tokens.weight torch.Size([32000, 4096])
model.layers.0.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.0.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.0.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.0.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.0.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.0.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.0.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.0.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.0.input_layernorm.weight torch.Size([4096])
model.layers.0.post_attention_layernorm.weight torch.Size([4096])
model.layers.1.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.1.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.1.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.1.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.1.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.1.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.1.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.1.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.1.input_layernorm.weight torch.Size([4096])
model.layers.1.post_attention_layernorm.weight torch.Size([4096])
model.layers.2.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.2.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.2.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.2.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.2.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.2.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.2.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.2.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.2.input_layernorm.weight torch.Size([4096])
model.layers.2.post_attention_layernorm.weight torch.Size([4096])
model.layers.3.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.3.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.3.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.3.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.3.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.3.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.3.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.3.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.3.input_layernorm.weight torch.Size([4096])
model.layers.3.post_attention_layernorm.weight torch.Size([4096])
model.layers.4.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.4.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.4.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.4.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.4.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.4.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.4.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.4.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.4.input_layernorm.weight torch.Size([4096])
model.layers.4.post_attention_layernorm.weight torch.Size([4096])
model.layers.5.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.5.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.5.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.5.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.5.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.5.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.5.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.5.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.5.input_layernorm.weight torch.Size([4096])
model.layers.5.post_attention_layernorm.weight torch.Size([4096])
model.layers.6.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.6.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.6.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.6.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.6.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.6.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.6.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.6.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.6.input_layernorm.weight torch.Size([4096])
model.layers.6.post_attention_layernorm.weight torch.Size([4096])
model.layers.7.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.7.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.7.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.7.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.7.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.7.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.7.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.7.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.7.input_layernorm.weight torch.Size([4096])
model.layers.7.post_attention_layernorm.weight torch.Size([4096])
model.layers.8.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.8.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.8.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.8.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.8.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.8.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.8.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.8.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.8.input_layernorm.weight torch.Size([4096])
model.layers.8.post_attention_layernorm.weight torch.Size([4096])
model.layers.9.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.9.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.9.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.9.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.9.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.9.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.9.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.9.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.9.input_layernorm.weight torch.Size([4096])
model.layers.9.post_attention_layernorm.weight torch.Size([4096])
model.layers.10.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.10.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.10.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.10.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.10.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.10.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.10.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.10.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.10.input_layernorm.weight torch.Size([4096])
model.layers.10.post_attention_layernorm.weight torch.Size([4096])
model.layers.11.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.11.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.11.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.11.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.11.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.11.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.11.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.11.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.11.input_layernorm.weight torch.Size([4096])
model.layers.11.post_attention_layernorm.weight torch.Size([4096])
model.layers.12.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.12.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.12.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.12.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.12.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.12.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.12.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.12.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.12.input_layernorm.weight torch.Size([4096])
model.layers.12.post_attention_layernorm.weight torch.Size([4096])
model.layers.13.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.13.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.13.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.13.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.13.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.13.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.13.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.13.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.13.input_layernorm.weight torch.Size([4096])
model.layers.13.post_attention_layernorm.weight torch.Size([4096])
model.layers.14.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.14.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.14.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.14.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.14.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.14.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.14.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.14.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.14.input_layernorm.weight torch.Size([4096])
model.layers.14.post_attention_layernorm.weight torch.Size([4096])
model.layers.15.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.15.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.15.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.15.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.15.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.15.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.15.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.15.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.15.input_layernorm.weight torch.Size([4096])
model.layers.15.post_attention_layernorm.weight torch.Size([4096])
model.layers.16.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.16.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.16.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.16.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.16.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.16.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.16.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.16.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.16.input_layernorm.weight torch.Size([4096])
model.layers.16.post_attention_layernorm.weight torch.Size([4096])
model.layers.17.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.17.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.17.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.17.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.17.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.17.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.17.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.17.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.17.input_layernorm.weight torch.Size([4096])
model.layers.17.post_attention_layernorm.weight torch.Size([4096])
model.layers.18.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.18.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.18.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.18.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.18.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.18.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.18.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.18.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.18.input_layernorm.weight torch.Size([4096])
model.layers.18.post_attention_layernorm.weight torch.Size([4096])
model.layers.19.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.19.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.19.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.19.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.19.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.19.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.19.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.19.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.19.input_layernorm.weight torch.Size([4096])
model.layers.19.post_attention_layernorm.weight torch.Size([4096])
model.layers.20.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.20.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.20.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.20.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.20.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.20.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.20.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.20.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.20.input_layernorm.weight torch.Size([4096])
model.layers.20.post_attention_layernorm.weight torch.Size([4096])
model.layers.21.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.21.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.21.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.21.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.21.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.21.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.21.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.21.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.21.input_layernorm.weight torch.Size([4096])
model.layers.21.post_attention_layernorm.weight torch.Size([4096])
model.layers.22.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.22.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.22.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.22.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.22.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.22.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.22.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.22.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.22.input_layernorm.weight torch.Size([4096])
model.layers.22.post_attention_layernorm.weight torch.Size([4096])
model.layers.23.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.23.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.23.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.23.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.23.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.23.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.23.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.23.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.23.input_layernorm.weight torch.Size([4096])
model.layers.23.post_attention_layernorm.weight torch.Size([4096])
model.layers.24.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.24.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.24.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.24.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.24.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.24.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.24.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.24.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.24.input_layernorm.weight torch.Size([4096])
model.layers.24.post_attention_layernorm.weight torch.Size([4096])
model.layers.25.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.25.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.25.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.25.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.25.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.25.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.25.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.25.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.25.input_layernorm.weight torch.Size([4096])
model.layers.25.post_attention_layernorm.weight torch.Size([4096])
model.layers.26.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.26.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.26.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.26.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.26.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.26.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.26.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.26.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.26.input_layernorm.weight torch.Size([4096])
model.layers.26.post_attention_layernorm.weight torch.Size([4096])
model.layers.27.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.27.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.27.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.27.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.27.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.27.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.27.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.27.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.27.input_layernorm.weight torch.Size([4096])
model.layers.27.post_attention_layernorm.weight torch.Size([4096])
model.layers.28.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.28.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.28.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.28.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.28.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.28.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.28.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.28.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.28.input_layernorm.weight torch.Size([4096])
model.layers.28.post_attention_layernorm.weight torch.Size([4096])
model.layers.29.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.29.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.29.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.29.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.29.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.29.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.29.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.29.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.29.input_layernorm.weight torch.Size([4096])
model.layers.29.post_attention_layernorm.weight torch.Size([4096])
model.layers.30.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.30.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.30.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.30.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.30.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.30.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.30.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.30.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.30.input_layernorm.weight torch.Size([4096])
model.layers.30.post_attention_layernorm.weight torch.Size([4096])
model.layers.31.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.31.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.31.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.31.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.31.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.31.mlp.gate_proj.weight torch.Size([11008, 4096])
model.layers.31.mlp.up_proj.weight torch.Size([11008, 4096])
model.layers.31.mlp.down_proj.weight torch.Size([4096, 11008])
model.layers.31.input_layernorm.weight torch.Size([4096])
model.layers.31.post_attention_layernorm.weight torch.Size([4096])
model.norm.weight torch.Size([4096])
lm_head.weight torch.Size([32000, 4096])

===================================================================
moe

model.embed_tokens.weight torch.Size([32000, 4096])
model.layers.0.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.0.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.0.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.0.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.0.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.0.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.0.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.0.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.0.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.0.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.0.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.0.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.0.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.0.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.0.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.0.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.0.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.0.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.0.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.0.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.0.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.0.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.0.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.0.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.0.input_layernorm.weight torch.Size([4096])
model.layers.0.post_attention_layernorm.weight torch.Size([4096])
model.layers.1.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.1.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.1.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.1.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.1.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.1.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.1.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.1.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.1.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.1.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.1.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.1.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.1.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.1.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.1.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.1.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.1.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.1.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.1.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.1.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.1.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.1.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.1.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.1.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.1.input_layernorm.weight torch.Size([4096])
model.layers.1.post_attention_layernorm.weight torch.Size([4096])
model.layers.2.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.2.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.2.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.2.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.2.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.2.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.2.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.2.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.2.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.2.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.2.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.2.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.2.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.2.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.2.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.2.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.2.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.2.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.2.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.2.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.2.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.2.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.2.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.2.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.2.input_layernorm.weight torch.Size([4096])
model.layers.2.post_attention_layernorm.weight torch.Size([4096])
model.layers.3.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.3.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.3.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.3.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.3.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.3.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.3.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.3.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.3.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.3.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.3.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.3.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.3.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.3.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.3.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.3.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.3.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.3.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.3.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.3.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.3.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.3.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.3.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.3.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.3.input_layernorm.weight torch.Size([4096])
model.layers.3.post_attention_layernorm.weight torch.Size([4096])
model.layers.4.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.4.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.4.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.4.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.4.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.4.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.4.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.4.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.4.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.4.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.4.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.4.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.4.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.4.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.4.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.4.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.4.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.4.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.4.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.4.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.4.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.4.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.4.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.4.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.4.input_layernorm.weight torch.Size([4096])
model.layers.4.post_attention_layernorm.weight torch.Size([4096])
model.layers.5.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.5.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.5.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.5.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.5.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.5.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.5.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.5.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.5.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.5.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.5.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.5.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.5.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.5.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.5.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.5.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.5.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.5.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.5.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.5.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.5.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.5.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.5.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.5.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.5.input_layernorm.weight torch.Size([4096])
model.layers.5.post_attention_layernorm.weight torch.Size([4096])
model.layers.6.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.6.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.6.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.6.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.6.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.6.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.6.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.6.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.6.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.6.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.6.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.6.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.6.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.6.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.6.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.6.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.6.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.6.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.6.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.6.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.6.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.6.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.6.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.6.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.6.input_layernorm.weight torch.Size([4096])
model.layers.6.post_attention_layernorm.weight torch.Size([4096])
model.layers.7.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.7.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.7.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.7.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.7.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.7.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.7.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.7.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.7.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.7.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.7.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.7.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.7.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.7.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.7.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.7.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.7.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.7.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.7.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.7.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.7.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.7.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.7.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.7.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.7.input_layernorm.weight torch.Size([4096])
model.layers.7.post_attention_layernorm.weight torch.Size([4096])
model.layers.8.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.8.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.8.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.8.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.8.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.8.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.8.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.8.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.8.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.8.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.8.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.8.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.8.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.8.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.8.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.8.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.8.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.8.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.8.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.8.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.8.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.8.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.8.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.8.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.8.input_layernorm.weight torch.Size([4096])
model.layers.8.post_attention_layernorm.weight torch.Size([4096])
model.layers.9.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.9.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.9.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.9.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.9.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.9.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.9.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.9.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.9.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.9.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.9.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.9.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.9.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.9.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.9.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.9.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.9.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.9.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.9.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.9.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.9.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.9.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.9.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.9.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.9.input_layernorm.weight torch.Size([4096])
model.layers.9.post_attention_layernorm.weight torch.Size([4096])
model.layers.10.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.10.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.10.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.10.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.10.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.10.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.10.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.10.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.10.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.10.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.10.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.10.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.10.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.10.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.10.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.10.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.10.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.10.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.10.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.10.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.10.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.10.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.10.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.10.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.10.input_layernorm.weight torch.Size([4096])
model.layers.10.post_attention_layernorm.weight torch.Size([4096])
model.layers.11.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.11.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.11.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.11.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.11.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.11.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.11.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.11.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.11.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.11.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.11.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.11.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.11.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.11.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.11.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.11.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.11.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.11.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.11.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.11.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.11.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.11.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.11.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.11.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.11.input_layernorm.weight torch.Size([4096])
model.layers.11.post_attention_layernorm.weight torch.Size([4096])
model.layers.12.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.12.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.12.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.12.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.12.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.12.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.12.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.12.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.12.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.12.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.12.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.12.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.12.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.12.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.12.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.12.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.12.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.12.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.12.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.12.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.12.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.12.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.12.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.12.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.12.input_layernorm.weight torch.Size([4096])
model.layers.12.post_attention_layernorm.weight torch.Size([4096])
model.layers.13.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.13.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.13.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.13.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.13.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.13.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.13.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.13.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.13.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.13.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.13.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.13.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.13.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.13.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.13.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.13.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.13.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.13.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.13.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.13.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.13.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.13.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.13.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.13.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.13.input_layernorm.weight torch.Size([4096])
model.layers.13.post_attention_layernorm.weight torch.Size([4096])
model.layers.14.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.14.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.14.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.14.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.14.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.14.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.14.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.14.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.14.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.14.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.14.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.14.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.14.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.14.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.14.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.14.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.14.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.14.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.14.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.14.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.14.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.14.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.14.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.14.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.14.input_layernorm.weight torch.Size([4096])
model.layers.14.post_attention_layernorm.weight torch.Size([4096])
model.layers.15.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.15.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.15.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.15.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.15.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.15.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.15.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.15.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.15.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.15.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.15.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.15.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.15.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.15.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.15.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.15.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.15.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.15.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.15.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.15.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.15.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.15.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.15.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.15.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.15.input_layernorm.weight torch.Size([4096])
model.layers.15.post_attention_layernorm.weight torch.Size([4096])
model.layers.16.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.16.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.16.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.16.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.16.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.16.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.16.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.16.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.16.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.16.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.16.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.16.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.16.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.16.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.16.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.16.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.16.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.16.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.16.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.16.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.16.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.16.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.16.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.16.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.16.input_layernorm.weight torch.Size([4096])
model.layers.16.post_attention_layernorm.weight torch.Size([4096])
model.layers.17.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.17.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.17.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.17.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.17.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.17.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.17.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.17.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.17.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.17.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.17.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.17.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.17.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.17.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.17.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.17.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.17.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.17.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.17.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.17.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.17.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.17.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.17.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.17.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.17.input_layernorm.weight torch.Size([4096])
model.layers.17.post_attention_layernorm.weight torch.Size([4096])
model.layers.18.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.18.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.18.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.18.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.18.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.18.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.18.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.18.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.18.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.18.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.18.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.18.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.18.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.18.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.18.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.18.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.18.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.18.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.18.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.18.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.18.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.18.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.18.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.18.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.18.input_layernorm.weight torch.Size([4096])
model.layers.18.post_attention_layernorm.weight torch.Size([4096])
model.layers.19.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.19.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.19.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.19.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.19.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.19.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.19.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.19.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.19.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.19.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.19.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.19.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.19.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.19.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.19.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.19.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.19.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.19.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.19.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.19.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.19.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.19.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.19.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.19.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.19.input_layernorm.weight torch.Size([4096])
model.layers.19.post_attention_layernorm.weight torch.Size([4096])
model.layers.20.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.20.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.20.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.20.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.20.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.20.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.20.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.20.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.20.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.20.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.20.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.20.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.20.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.20.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.20.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.20.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.20.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.20.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.20.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.20.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.20.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.20.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.20.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.20.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.20.input_layernorm.weight torch.Size([4096])
model.layers.20.post_attention_layernorm.weight torch.Size([4096])
model.layers.21.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.21.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.21.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.21.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.21.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.21.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.21.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.21.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.21.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.21.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.21.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.21.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.21.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.21.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.21.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.21.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.21.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.21.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.21.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.21.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.21.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.21.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.21.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.21.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.21.input_layernorm.weight torch.Size([4096])
model.layers.21.post_attention_layernorm.weight torch.Size([4096])
model.layers.22.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.22.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.22.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.22.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.22.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.22.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.22.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.22.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.22.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.22.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.22.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.22.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.22.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.22.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.22.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.22.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.22.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.22.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.22.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.22.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.22.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.22.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.22.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.22.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.22.input_layernorm.weight torch.Size([4096])
model.layers.22.post_attention_layernorm.weight torch.Size([4096])
model.layers.23.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.23.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.23.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.23.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.23.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.23.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.23.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.23.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.23.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.23.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.23.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.23.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.23.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.23.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.23.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.23.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.23.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.23.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.23.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.23.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.23.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.23.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.23.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.23.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.23.input_layernorm.weight torch.Size([4096])
model.layers.23.post_attention_layernorm.weight torch.Size([4096])
model.layers.24.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.24.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.24.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.24.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.24.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.24.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.24.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.24.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.24.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.24.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.24.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.24.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.24.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.24.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.24.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.24.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.24.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.24.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.24.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.24.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.24.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.24.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.24.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.24.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.24.input_layernorm.weight torch.Size([4096])
model.layers.24.post_attention_layernorm.weight torch.Size([4096])
model.layers.25.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.25.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.25.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.25.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.25.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.25.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.25.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.25.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.25.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.25.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.25.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.25.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.25.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.25.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.25.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.25.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.25.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.25.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.25.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.25.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.25.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.25.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.25.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.25.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.25.input_layernorm.weight torch.Size([4096])
model.layers.25.post_attention_layernorm.weight torch.Size([4096])
model.layers.26.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.26.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.26.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.26.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.26.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.26.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.26.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.26.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.26.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.26.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.26.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.26.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.26.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.26.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.26.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.26.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.26.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.26.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.26.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.26.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.26.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.26.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.26.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.26.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.26.input_layernorm.weight torch.Size([4096])
model.layers.26.post_attention_layernorm.weight torch.Size([4096])
model.layers.27.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.27.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.27.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.27.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.27.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.27.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.27.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.27.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.27.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.27.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.27.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.27.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.27.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.27.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.27.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.27.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.27.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.27.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.27.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.27.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.27.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.27.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.27.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.27.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.27.input_layernorm.weight torch.Size([4096])
model.layers.27.post_attention_layernorm.weight torch.Size([4096])
model.layers.28.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.28.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.28.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.28.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.28.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.28.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.28.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.28.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.28.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.28.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.28.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.28.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.28.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.28.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.28.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.28.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.28.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.28.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.28.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.28.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.28.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.28.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.28.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.28.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.28.input_layernorm.weight torch.Size([4096])
model.layers.28.post_attention_layernorm.weight torch.Size([4096])
model.layers.29.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.29.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.29.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.29.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.29.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.29.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.29.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.29.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.29.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.29.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.29.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.29.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.29.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.29.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.29.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.29.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.29.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.29.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.29.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.29.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.29.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.29.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.29.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.29.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.29.input_layernorm.weight torch.Size([4096])
model.layers.29.post_attention_layernorm.weight torch.Size([4096])
model.layers.30.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.30.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.30.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.30.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.30.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.30.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.30.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.30.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.30.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.30.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.30.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.30.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.30.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.30.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.30.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.30.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.30.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.30.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.30.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.30.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.30.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.30.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.30.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.30.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.30.input_layernorm.weight torch.Size([4096])
model.layers.30.post_attention_layernorm.weight torch.Size([4096])
model.layers.31.self_attn.q_proj.weight torch.Size([4096, 4096])
model.layers.31.self_attn.k_proj.weight torch.Size([4096, 4096])
model.layers.31.self_attn.v_proj.weight torch.Size([4096, 4096])
model.layers.31.self_attn.o_proj.weight torch.Size([4096, 4096])
model.layers.31.self_attn.rotary_emb.inv_freq torch.Size([64])
model.layers.31.mlp.gate.gate_network.0.weight torch.Size([16, 4096])
model.layers.31.mlp.gate.gate_network.2.weight torch.Size([16, 16])
model.layers.31.mlp.gate.weight_noise.weight torch.Size([16, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.0 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.1 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.2 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.3 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.4 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.5 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.6 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.7 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.8 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.9 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.10 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.11 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.12 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.13 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.14 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_gate.15 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.0 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.1 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.2 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.3 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.4 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.5 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.6 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.7 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.8 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.9 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.10 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.11 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.12 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.13 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.14 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_up.15 torch.Size([688, 4096])
model.layers.31.mlp.calculator.experts.weight_down.0 torch.Size([4096, 688])
model.layers.31.mlp.calculator.experts.weight_down.1 torch.Size([4096, 688])
model.layers.31.mlp.calculator.experts.weight_down.2 torch.Size([4096, 688])
model.layers.31.mlp.calculator.experts.weight_down.3 torch.Size([4096, 688])
model.layers.31.mlp.calculator.experts.weight_down.4 torch.Size([4096, 688])
model.layers.31.mlp.calculator.experts.weight_down.5 torch.Size([4096, 688])
model.layers.31.mlp.calculator.experts.weight_down.6 torch.Size([4096, 688])
model.layers.31.mlp.calculator.experts.weight_down.7 torch.Size([4096, 688])
model.layers.31.mlp.calculator.experts.weight_down.8 torch.Size([4096, 688])
model.layers.31.mlp.calculator.experts.weight_down.9 torch.Size([4096, 688])
model.layers.31.mlp.calculator.experts.weight_down.10 torch.Size([4096, 688])
model.layers.31.mlp.calculator.experts.weight_down.11 torch.Size([4096, 688])
model.layers.31.mlp.calculator.experts.weight_down.12 torch.Size([4096, 688])
model.layers.31.mlp.calculator.experts.weight_down.13 torch.Size([4096, 688])
model.layers.31.mlp.calculator.experts.weight_down.14 torch.Size([4096, 688])
model.layers.31.mlp.calculator.experts.weight_down.15 torch.Size([4096, 688])
model.layers.31.input_layernorm.weight torch.Size([4096])
model.layers.31.post_attention_layernorm.weight torch.Size([4096])
model.norm.weight torch.Size([4096])
lm_head.weight torch.Size([32000, 4096])
"""
